'''
Created on Aug 28, 2012

@author: kingsfield
'''
from util import Util
import math

__mydebug__ = False

class ContextCF(object):
    '''
    classdocs
    '''
    wordmap = dict()
    itemmap = dict()
    catmap = dict()
    catmapinv = dict()
    itemmapinv = dict()
    wordmapinv = dict()
    
    word_sim = dict()
    word_item_score = dict()
    word_count = dict()
    word_item_count = dict()
    cat_word_map = dict()
    cat_item_map = dict()
    cat_item_count = dict()
    max_wid = 59168
    max_iid = 69858
    with open('/home/kingsfield/Desktop/kaggle/kaggle/idmap/wordmap') as fr:
        data = fr.readlines()
        for line in data:
            ss = line.split(' ')
            wordmap[ss[0]] = int(ss[1])
            wordmapinv[int(ss[1])] = ss[0]
    with open('/home/kingsfield/Desktop/kaggle/kaggle/idmap/itemmap') as fr:
        data = fr.readlines()
        for line in data:
            ss = line.split(' ')
            itemmap[ss[0]] = int(ss[1])
            itemmapinv[int(ss[1])] = ss[0]
    with open('/home/kingsfield/Desktop/kaggle/kaggle/idmap/categorymap') as fr:
        data = fr.readlines()
        for line in data:
            ss = line.split(' ')
            catmap[ss[0]] = int(ss[1])
            catmapinv[int(ss[1])] = ss[0]
    with open('/home/kingsfield/Desktop/kaggle/kaggle/idmap/catwordmap') as fr:
        data = fr.readlines()
        for line in data:
            ss = line.split(' ')
            cid = int(ss[0])
            wid = int(ss[1])
            if not cid in cat_word_map:
                cat_word_map[cid] = set()
            cat_word_map[cid].add(wid) 
    with open('/home/kingsfield/Desktop/kaggle/kaggle/idmap/catitemmap') as fr:
        data = fr.readlines()
        for line in data:
            ss = line.split(' ')
            cid = int(ss[0])
            iid = int(ss[1])
            count = int(ss[2])
            if not cid in cat_item_map:
                cat_item_map[cid] = set()
                cat_item_count[cid] = dict()
            cat_item_map[cid].add(iid)
            cat_item_count[cid][iid] = count

    def __init__(self, params):
        ''
        
    @classmethod
    def train(cls):
        filepath = '/home/kingsfield/Desktop/kaggle/kaggle/train.csv'
        sum_line = 1865270
        with open(filepath) as fr:
            line = fr.readline()
            comp = dict()
            margin = dict()
            sum_comp = 0
            line_count = 0
            step = sum_line / 100
            while True:
                line_count += 1
                if __mydebug__:
                    if line_count / step >= 5:
                        break
                if line_count % step == 0:
                    print '--%d' % (line_count / step)
                line = fr.readline()
                if not line:
                    break
                ss = line.split(',')
                iid = cls.itemmap[ss[1]]
                query = ss[3]
                words = Util.wordutil.getwords(query)
                wids = [cls.wordmap[w] for w in words if w in cls.wordmap]
                for idx in xrange(len(wids)):
                    key = cls._get_pair(wids[idx], iid)
                    cls.word_item_score[key] = cls.word_item_score.get(key, 0) + 1
                    cls.word_count[wids[idx]] = cls.word_count.get(wids[idx], 0) + 1
                    if not wids[idx] in cls.word_item_count: 
                        cls.word_item_count[wids[idx]] = dict()
                    cls.word_item_count[wids[idx]][iid] = cls.word_item_count[wids[idx]].get(iid, 0) + 1 
                    for jdx in xrange(idx + 1, len(wids)):
                        margin[wids[idx]] = margin.get(wids[idx], 0) + 1
                        margin[wids[jdx]] = margin.get(wids[jdx], 0) + 1
                        comp_k = cls._get_pair(wids[idx], wids[jdx])
                        comp[comp_k] = comp.get(comp_k, 0) + 1
                        sum_comp += 1
                
            for k in comp:
                u, v = k.split(',')
                u = int(u)
                v = int(v)
                H1 = margin[u] * 1.0 / sum_comp
                H1 = -1.0 * H1 * math.log(H1)
                H2 = margin[v] * 1.0 / sum_comp
                H2 = -1.0 * H2 * math.log(H2)
                H12 = comp[k] * 1.0 / sum_comp
                H12 = -1.0 * H12 * math.log(H12)
                sim = H1 + H2 - H12
                cls.word_sim[k] = sim
            del comp
            del margin

    @classmethod
    def _get_pair(cls, u, v):
        if u < v:
            return str(u) + ',' + str(v)
        else:
            return str(v) + ',' + str(u)

    @classmethod
    def cf_sim_word(cls, words, cat):
        'words is a list of int'
        simrank = dict()
        cid = cls.catmap[cat]
        for w1 in words:
            for w2 in cls.cat_word_map[cid]:
                if w1 != w2:
                    key = cls._get_pair(w1, w2)
                    if key in cls.word_sim:
                        simrank[w2] = simrank.get(w2, 0.0) + cls.word_sim[key]
        for w in words:
            simrank[w] = 1.0
        return simrank
    
    @classmethod
    def rec_items(cls, words, cat, topk):
        'words is a list of int'
        if not cat in cls.catmap:
            return ['1854151', '2416092', '9830614', '2262047', '2345191']
        simrank_limit = 20
        iids_limit = 100
        simrank = cls.cf_sim_word(words, cat)
        #print '[debug]:len simrand=%d' % len(simrank)
        res = dict()
        cid = cls.catmap[cat]
        iids = cls.cat_item_map[cid]
        if len(iids) * len(simrank) > simrank_limit * iids_limit:
            simrank = sorted(simrank.items(), key=lambda x:x[1], reverse=True)
            if len(simrank) >= simrank_limit:
                simrank = simrank[0:simrank_limit] 
            simrank = dict(simrank)
            iids = sorted(cls.cat_item_count[cid].items(), key=lambda x:x[1], reverse=True)
            if len(iids) >= iids_limit:
                iids = iids[0:iids_limit]
            iids = dict(iids)
        #print '[debug]:len iids%s' % len(iids)
        for iid in iids:
            ctr = 0.0
            for w in simrank:
                ctr += cls.word_item_score.get(cls._get_pair(w, iid), 0.0) / cls.word_count[w] 
            res[iid] = ctr
        rank = sorted(res.items(), key=lambda x: x[1], reverse=True)
        res = [i[0] for i in rank[0:topk]]
        res = [cls.itemmapinv[i] for i in res if i in cls.itemmapinv]
        return res

    @classmethod
    def test(cls):
        topk = 5
        with open('/home/kingsfield/Desktop/kaggle/kaggle/test.csv') as fr:
            with open('/home/kingsfield/Desktop/kaggle/kaggle/out/out0829a.csv', 'w') as fw:
                fw.write('sku\n')
                line = fr.readline()
                idx = 0
                sum_line = 1243511
                step = sum_line / 1000
                while True:
                    line = fr.readline()
                    idx += 1
                    #print idx
                    if idx % step == 0:
                        print '----%d' % (idx / step)
                    if not line:
                        break
                    ss = line.split(',')
                    cat = ss[1]
                    query = ss[2]
                    words = Util.wordutil.getwords(query) 
                    wids = [cls.wordmap[w] for w in words if w in cls.wordmap]
                    rec = cls.rec_items(wids, cat, topk)
                    outline = ' '.join(rec) + '\n'
                    fw.write(outline)

if __name__ == '__main__':
    ContextCF.train()
    print 'begin to test'
    ContextCF.test()

#    topk = 100
#    query = 'phantom of the opera'
#    words = Util.wordutil.getwords(query) 
#    wids = [ContextCF.wordmap[w] for w in words if w in ContextCF.wordmap]
#    rec = ContextCF.rec_items(wids, 'cat02015', topk)
#    print '[debug]:origin at %s' % rec.index('2125233')
#    print rec
#    
#    query = 'Blue tooth headphones'
#    words = Util.wordutil.getwords(query) 
#    wids = [ContextCF.wordmap[w] for w in words if w in ContextCF.wordmap]
#    rec = ContextCF.rec_items(wids, 'pcmcat171900050028', topk)
#    print '[debug]:origin at %s' % rec.index('2264036')
#    print rec

#    query = 'Televisiones Panasonic  50 pulgadas'
#    words = Util.wordutil.getwords(query) 
#    wids = [ContextCF.wordmap[w] for w in words if w in ContextCF.wordmap]
#    sims = ContextCF.cf_sim_word(wids, 'abcat0101001')
#    simwords = [[ContextCF.wordmapinv[wid], sims[wid]] for wid in sims]
#    rank = sorted(simwords, key=lambda x:x[1], reverse=True)
#    print str(rank[0:20])
#    
#    query = 'AC power cord'
#    words = Util.wordutil.getwords(query) 
#    wids = [ContextCF.wordmap[w] for w in words if w in ContextCF.wordmap]
#    sims = ContextCF.cf_sim_word(wids, 'abcat0101001')
#    simwords = [[ContextCF.wordmapinv[wid], sims[wid]] for wid in sims]
#    rank = sorted(simwords, key=lambda x:x[1], reverse=True)
#    print str(rank[0:20])

#    cat = 'abcat0101001'
#    cid = ContextCF.catmap[cat]
#    catwids = ContextCF.cat_word_map[cid]
#    words = [ContextCF.wordmapinv[wid] for wid in catwids]
#    print 'len=%d' % len(words)
#    print words
    print 'done'
